package org.tulkas.robot;

import java.util.ArrayList;
import java.util.BitSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.lexer.Lexer;
import org.htmlparser.lexer.Page;
import org.htmlparser.nodes.TagNode;
import org.htmlparser.nodes.TextNode;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.tulkas.robot.patch.CssSelectorNodeFilter;

/**
 * A Refiner is a tool for extracting the content from one or more strings. It can also do some post processing of the content.
 * The extraction can be done by the regexp or a simple template.
 * The rule of the template:
 * 1. (*) stands for 0 or more any characters
 * 2. (_) stands for 0 or more whitespace characters
 * 3. \ is used for escaping. Any character after \ is treated as the original character. 
 * 		e.g. (\*) could be used to represent the actual '(*)' string instead of not any characters.   
 * 
 * @author <a href="mailto:tulkas.web@gmail.com">Tulkas</a>
 *
 */
public class Refiner implements Cloneable {
	private static Logger logger = LoggerFactory.getLogger(Refiner.class);

	private Set<String> strings = new LinkedHashSet<String>(1);
	private List<TagNode> tagNodes = new ArrayList<TagNode>();
	private final TemplateParser parser = new TemplateParser();

	public Refiner(String content) {
		if (content != null)
			strings.add(content);
	}

	public Refiner(String[] contents) {
		if (contents != null) {
			for (String str : contents)
				strings.add(str);
		}
	}

	private Pattern buildPattern(String template) {
		String regexp = parser.parse(template);
		return Pattern.compile(regexp, Pattern.DOTALL
				| Pattern.CASE_INSENSITIVE);
	}

	private Pattern buildPattern(String startTpl, String endTpl) {
		String start = parser.parse(startTpl);
		String end = parser.parse(endTpl);
		return Pattern.compile(start + "(.*?)" + end, Pattern.DOTALL
				| Pattern.CASE_INSENSITIVE);
	}

	public Refiner extract(String template) {
		return extractByRegexp(buildPattern(template));
	}

	public Refiner extractFirst(String template) {
		return extractFirstByRegexp(buildPattern(template));
	}

	public Refiner extractByRegexp(Pattern pattern) {
		return extractByRegexp(pattern, 0);
	}

	public Refiner extractFirstByRegexp(Pattern pattern) {
		return extractFirstByRegexp(pattern, 0);
	}

	private LinkedHashSet<String> extractByRegexpInternel(Pattern pattern,
			int groupIndex) {
		LinkedHashSet<String> set = new LinkedHashSet<String>();
		for (String str : strings) {
			Matcher mt = pattern.matcher(str);
			while (mt.find()) {
				set.add(mt.group(groupIndex));
			}
		}
		return set;
	}

	public Refiner extractByRegexp(Pattern pattern, int groupIndex) {
		strings = extractByRegexpInternel(pattern, groupIndex);
		return this;
	}

	public Refiner extractFirstByRegexp(Pattern pattern, int groupIndex) {
		LinkedHashSet<String> set = extractByRegexpInternel(pattern, groupIndex);
		String str = null;
		strings.clear();
		if (set.size() > 0) {
			str = set.iterator().next();
			strings.add(str);
		}
		return this;
	}

	public Refiner extractBetween(String startTpl, String endTpl) {
		Pattern pt = buildPattern(startTpl, endTpl);
		return extractByRegexp(pt, 1);
	}

	public Refiner extractFirstBetween(String startTpl, String endTpl) {
		Pattern pt = buildPattern(startTpl, endTpl);
		return extractFirstByRegexp(pt, 1);
	}

	public Refiner replace(String template, String replacement) {
		return replaceByRegexp(buildPattern(template), replacement);
	}

	public Refiner replaceByRegexp(Pattern regexp, String replacement) {
		Set<String> ret = new LinkedHashSet<String>(strings.size());
		for (String str : strings) {
			ret.add(str.replaceAll(regexp.pattern(), replacement));
		}
		strings = ret;
		return this;
	}

	public Refiner extractTags(String cssSelector) {
		NodeFilter filter = new CssSelectorNodeFilter(cssSelector);
		Set<String> list = new LinkedHashSet<String>();
		tagNodes.clear();
		for (String str : strings) {
			Lexer lexer = new Lexer(new Page(str));
			Parser parser = new Parser(lexer, Parser.DEVNULL);
			try {
				NodeList nl = parser.extractAllNodesThatMatch(filter);
				for (int j = 0; j < nl.size(); j++) {
					Node node = nl.elementAt(j);
					if (node instanceof TagNode) {
						TagNode tn = (TagNode) node;
						tagNodes.add(tn);
						int s = node.getStartPosition();
						int e = node.getEndPosition();
						if (tn.getEndTag() != null) {
							e = tn.getEndTag().getEndPosition();
						}
						list.add(str.substring(s, e));
					}
				}
			} catch (ParserException e) {
				logger.debug("Error extracting tags with cssSelector ["
						+ cssSelector + "]. " + e.toString());
			}
		}
		strings = list;
		return this;
	}

	public Refiner extractAttributesFromTags(String attributeName) {
		strings.clear();
		for (TagNode tn : tagNodes) {
			String v = tn.getAttribute(attributeName);
			if (v != null)
				strings.add(v);
		}
		return this;
	}

	public Refiner extractTextsFromTags() {
		strings.clear();
		for (TagNode tn : tagNodes) {
			NodeList nl2 = tn.getChildren();
			StringBuffer sb = new StringBuffer();
			for (int k = 0; k < nl2.size(); k++) {
				Node nc = nl2.elementAt(k);
				if (nc instanceof TextNode) {
					String v = ((TextNode) nc).getText();
					if (v != null) {
						sb.append(v);
					}
				}
			}
			String va = sb.toString();
			if (!"".equals(va))
				strings.add(va);
		}
		return this;
	}

	public Refiner removeTags(String cssSelector) {
		NodeFilter filter = new CssSelectorNodeFilter(cssSelector);
		Set<String> list = new LinkedHashSet<String>(strings.size());
		for (String str : strings) {
			Lexer lexer = new Lexer(new Page(str));
			Parser parser = new Parser(lexer, Parser.DEVNULL);
			BitSet bits = new BitSet(str.length());
			try {
				NodeList nl = parser.extractAllNodesThatMatch(filter);
				for (int j = 0; j < nl.size(); j++) {
					Node node = nl.elementAt(j);
					if (node instanceof TagNode) {
						TagNode tn = (TagNode) node;
						int s = node.getStartPosition();
						int e = node.getEndPosition();
						if (tn.getEndTag() != null) {
							e = tn.getEndTag().getEndPosition();
						}
						bits.set(s, e, true);
					}

				}
			} catch (ParserException e) {
				logger.debug("Error removing tags with cssSelector ["
						+ cssSelector + "]. " + e.toString());
			}
			StringBuffer sb = new StringBuffer();
			for (int i = 0; i < str.length(); i++) {
				if (!bits.get(i)) {
					sb.append(str.charAt(i));
				}
			}
			list.add(sb.toString());
		}
		strings = list;
		return this;
	}

	public Refiner trim() {
		Set<String> list = new LinkedHashSet<String>(strings.size());
		for (String str : strings) {
			String str2 = str.trim();
			if (str2 != null && !"".equals(str2))
				list.add(str2);
		}
		strings = list;
		return this;
	}

	public Refiner clone() {
		try {
			Refiner ret = (Refiner) super.clone();
			ret.strings = new LinkedHashSet<String>(strings);
			ret.tagNodes = new ArrayList<TagNode>(tagNodes);
			return ret;
		} catch (CloneNotSupportedException e) {
			return null;
		}
	}

	public String getResult() {
		if (strings.size() > 0)
			return strings.iterator().next();
		else
			return null;
	}

	public TagNode[] getResultTags() {
		TagNode[] ret = new TagNode[tagNodes.size()];
		tagNodes.toArray(ret);
		return ret;
	}

	public String[] getResults() {
		String[] ret = new String[strings.size()];
		strings.toArray(ret);
		return ret;
	}

	class TemplateParser {
		static final int NORMAL = 1;
		static final int ESCAPE = 2;
		static final int S = 3;
		static final int S_ANY = 4;
		static final int S_WHITESPACE = 5;

		private String in;
		private StringBuffer out;
		private int status;
		private boolean in_normal;
		private int cur;

		private void processNormal(Character c) {
			if (!in_normal) {
				out.append("\\Q");
				in_normal = true;
			}
			out.append(c);
		}

		private void processNormal(String s) {
			if (!in_normal) {
				out.append("\\Q");
				in_normal = true;
			}
			out.append(s);
		}

		private void processSpecial(String exp) {
			if (in_normal) {
				out.append("\\E");
				in_normal = false;
			}
			out.append(exp);
		}

		public void processChar(Character c) {
			switch (status) {
			case NORMAL:
				if (c == '\\') {
					status = ESCAPE;
				} else if (c == '(') {
					status = S;
				} else {
					processNormal(c);
				}
				break;
			case ESCAPE:
				processNormal(c);
				status = NORMAL;
				break;
			case S:
				if (c == '*') {
					status = S_ANY;
				} else if (c == '_') {
					status = S_WHITESPACE;
				} else if (c == '\\') {
					processNormal('(');
					status = ESCAPE;
				} else if (c == '(') {
					processNormal('(');
				} else {
					processNormal("(" + c);
					status = NORMAL;
				}
				break;
			case S_ANY:
				if (c == ')') {
					processSpecial(".*?");
					status = NORMAL;
				} else if (c == '\\') {
					processNormal("(*");
					status = ESCAPE;
				} else {
					processNormal("(*" + c);
					status = NORMAL;
				}
				break;
			case S_WHITESPACE:
				if (c == ')') {
					processSpecial("\\s*?");
					status = NORMAL;
				} else if (c == '\\') {
					processNormal("(_");
					status = ESCAPE;
				} else {
					processNormal("(_" + c);
					status = NORMAL;
				}
				break;
			}
		}

		public String parse(String input) {
			in = input;
			out = new StringBuffer();
			status = NORMAL;
			in_normal = false;
			cur = 0;

			for (int i = 0; i < in.length(); i++) {
				Character c = in.charAt(cur);
				processChar(c);
				cur++;
			}
			if (in_normal)
				out.append("\\E");
			return out.toString();
		}
	}
}
